{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "source": [
    "# word2vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "source": [
    "#### for word2vec:\n",
    "https://towardsdatascience.com/implementing-word2vec-in-pytorch-skip-gram-model-e6bae040d2fb\n",
    "#### for transD:\n",
    "https://github.com/jimmywangheng/knowledge_representation_pytorch/\n",
    "#### for RNN:\n",
    "https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T07:49:26.757074Z",
     "start_time": "2019-07-06T07:49:25.645816Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "from collections import defaultdict, Counter\n",
    "import networkx as nx\n",
    "import numpy as np\n",
    "import pylab as plt\n",
    "import random\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from copy import deepcopy\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T07:57:06.878425Z",
     "start_time": "2019-07-06T07:57:06.520703Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "# parameters and data\n",
    "vocabulary_size = 10\n",
    "embedding_dims = 2\n",
    "sample_size = 10000\n",
    "#pairs=list(zip(range(9),range(1,10)))+[(9,0)]+list(zip(range(10),range(10)))\n",
    "#pairs=list(zip(range(9),range(1,10)))\n",
    "pairs=list(zip(range(9),range(1,10)))+list(zip(range(10),range(10)))\n",
    "idx_pairs=np.array(random.choices(pairs,k=sample_size))\n",
    "idx_pairs = np.apply_along_axis(np.random.permutation,1,idx_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T07:57:08.316902Z",
     "start_time": "2019-07-06T07:57:08.312666Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 1),\n",
       " (1, 2),\n",
       " (2, 3),\n",
       " (3, 4),\n",
       " (4, 5),\n",
       " (5, 6),\n",
       " (6, 7),\n",
       " (7, 8),\n",
       " (8, 9),\n",
       " (0, 0),\n",
       " (1, 1),\n",
       " (2, 2),\n",
       " (3, 3),\n",
       " (4, 4),\n",
       " (5, 5),\n",
       " (6, 6),\n",
       " (7, 7),\n",
       " (8, 8),\n",
       " (9, 9)]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T07:57:19.771879Z",
     "start_time": "2019-07-06T07:57:19.768794Z"
    },
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "def oneHot(word_idx):\n",
    "    x = torch.zeros(vocabulary_size).float()\n",
    "    x[word_idx] = 1.0\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T08:01:33.182914Z",
     "start_time": "2019-07-06T07:57:50.299897Z"
    },
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loss at epo 0: 1.399356114923954\n",
      "Loss at epo 2: 1.0292332652807237\n",
      "Loss at epo 4: 1.0267999783039092\n",
      "Loss at epo 6: 1.025856315755844\n",
      "Loss at epo 8: 1.0252606665611268\n",
      "Loss at epo 10: 1.0248311063289641\n",
      "Loss at epo 12: 1.0244993392944335\n",
      "Loss at epo 14: 1.0242323592662812\n",
      "Loss at epo 16: 1.0240114040851593\n",
      "Loss at epo 18: 1.0238249147891998\n",
      "Loss at epo 20: 1.0236650954723359\n",
      "Loss at epo 22: 1.0235264299869538\n",
      "Loss at epo 24: 1.023404911518097\n",
      "Loss at epo 26: 1.0232975048542023\n",
      "Loss at epo 28: 1.0232018991470337\n",
      "Loss at epo 30: 1.0231163442611695\n",
      "Loss at epo 32: 1.023039317560196\n",
      "Loss at epo 34: 1.022969713783264\n",
      "Loss at epo 36: 1.022906504392624\n",
      "Loss at epo 38: 1.02284889626503\n",
      "Loss at epo 40: 1.0227962481498718\n",
      "Loss at epo 42: 1.022747976541519\n",
      "Loss at epo 44: 1.0227035741329193\n",
      "Loss at epo 46: 1.0226626495361328\n",
      "Loss at epo 48: 1.0226248146533965\n",
      "Loss at epo 50: 1.022589761543274\n",
      "Loss at epo 52: 1.0225572061061858\n",
      "Loss at epo 54: 1.0225269274711608\n",
      "Loss at epo 56: 1.0224987321853638\n",
      "Loss at epo 58: 1.022472420358658\n",
      "Loss at epo 60: 1.0224478448867798\n",
      "Loss at epo 62: 1.0224248598575592\n",
      "Loss at epo 64: 1.022403295135498\n",
      "Loss at epo 66: 1.0223830512046814\n",
      "Loss at epo 68: 1.0223639968395233\n",
      "Loss at epo 70: 1.0223460433483125\n",
      "Loss at epo 72: 1.0223290714263915\n",
      "Loss at epo 74: 1.0223130489349366\n",
      "Loss at epo 76: 1.0222979133605956\n",
      "Loss at epo 78: 1.0222835556030274\n",
      "Loss at epo 80: 1.0222699581623078\n",
      "Loss at epo 82: 1.0222570275783538\n",
      "Loss at epo 84: 1.0222447298049926\n",
      "Loss at epo 86: 1.0222330652713776\n",
      "Loss at epo 88: 1.0222219492435456\n",
      "Loss at epo 90: 1.022211360025406\n",
      "Loss at epo 92: 1.0222012673377991\n",
      "Loss at epo 94: 1.0221915943622588\n",
      "Loss at epo 96: 1.0221823862552644\n",
      "Loss at epo 98: 1.0221735826969147\n"
     ]
    }
   ],
   "source": [
    "# model and training\n",
    "\n",
    "W1 = Variable(torch.randn(embedding_dims, vocabulary_size).float(),\n",
    "               requires_grad=True)\n",
    "W2 = Variable(torch.randn(vocabulary_size, embedding_dims).float(),\n",
    "               requires_grad=True)\n",
    "#W1 = Variable(W1_data.t(),\n",
    "#              requires_grad=True)\n",
    "#W2 = Variable(W1_data,\n",
    "#              requires_grad=True)\n",
    "\n",
    "num_epochs = 100\n",
    "learning_rate = 0.01\n",
    "Loss=[]\n",
    "for epo in range(num_epochs):\n",
    "    loss_val = 0\n",
    "    for context, target in idx_pairs:\n",
    "        x = Variable(oneHot(context)).float()\n",
    "        y_true = Variable(torch.from_numpy(np.array([target])).long())\n",
    "\n",
    "        z1 = torch.matmul(W1, x)\n",
    "        z2 = torch.matmul(W2, z1)\n",
    "    \n",
    "        log_softmax = F.log_softmax(z2, dim=0)\n",
    "        loss = F.nll_loss(log_softmax.view(1,-1), y_true)\n",
    "        loss_val += loss.item()\n",
    "        loss.backward()\n",
    "        W1.data -= learning_rate * W1.grad.data\n",
    "        W2.data -= learning_rate * W2.grad.data\n",
    "\n",
    "        W1.grad.data.zero_()\n",
    "        W2.grad.data.zero_()\n",
    "    if epo % 2 == 0:    \n",
    "        print(f'Loss at epo {epo}: {loss_val/len(idx_pairs)}')\n",
    "    Loss.append(loss_val/len(idx_pairs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T08:02:09.002127Z",
     "start_time": "2019-07-06T08:02:08.864638Z"
    },
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Loss')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZEAAAEPCAYAAACDTflkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHW5JREFUeJzt3X+cHXV97/HXe2cJXAhCQhYiCTEoXBErII9tCoXW0MsDg/oA1FoJovgjjX2IUm/Fqq3XtNBWcwtqfaBCbhoDKLFKpc0VRIEKuVxE2CDllyC5iLIE2ZWFQBAD2XzuHzMnHHbPzOw52fNjz76fj8d57Dkzc2a+k4F973c+M99RRGBmZtaInnY3wMzMpi6HiJmZNcwhYmZmDXOImJlZwxwiZmbWMIeImZk1zCFiZmYNc4iYmVnDHCJmZtaw3nY3oNnmzJkTCxcubHczzMymjI0bN/46IvomsmzXh8jChQsZGBhodzPMzKYMSb+Y6LI+nWVmZg1ziJiZWcMcImZm1jCHiJmZNazlISJpjaQhSfeULPe7kkYl/XHVtLMkPZi9zmp+a83MrEg7rs5aC1wEXJa3gKQEWAl8v2rabGAF0A8EsFHS+oh4crIbeP1tW1m9fgvDI6P0zU5Ydso+nLho5mRvxsxsymt5TyQiNgAjJYt9BPhXYKhq2huB6yJiJAuO64Alk92+62/byoVXPMnQyCgBDI2McuEVT3L9bVsne1NmZlNex9VEJM0D3gpcPGbWPOCRqs+D2bRJtXr9FrY9/9JHBm97Pli9fstkb8rMbMrruBABvgh8IiJGx0xXjWVrPiBe0nJJA5IGhoeH69r48MjYzRZPNzObzjrxjvV+4JuSAOYAb5K0nbTnsbhqufnAjbVWEBGrgFUA/f39NYMmT9/shKEagdE3O6lnNWZm00LH9UQi4uCIWBgRC4ErgQ9FxL+RFtlPkjRL0izgJKoK75Nl2Sn7sPuMl3Z6dp8hlp2yz2Rvysxsymt5T0TSOtIexRxJg6RXXO0GEBFj6yA7RcSIpPOB27NJ50VEWYG+bpWrsD536Qg7Avb31VlmZrkUUdfZnimnv78/GhmA8U/+6lEWHb4H5565XxNaZWbWuSRtjIj+iSzbcaezOkXSA9tdSzczK+QQydGbiNEd3d1LMzPbVQ6RHEkPjLonYmZWyCGSI3FPxMyslEMkR9IDozva3Qozs87mEMmR9IjRUfdEzMyKOERyJImvzjIzK+MQyeGaiJlZOYdIDtdEzMzKOURyuCZiZlbOIZIjSdwTMTMr4xDJkfS4JmJmVsYhksN3rJuZlXOI5OhNXBMxMyvjEMnhmoiZWTmHSA7XRMzMyjlEcrgmYmZWrqUhImmNpCFJ9+TMP1XSXZLulDQg6fiqeaPZ9DslrW92W33HuplZuVY/Y30tcBFwWc78G4D1ERGSjgC+BRyWzXsuIo5qfhNTfrKhmVm5lvZEImIDMFIwf2u8+ND3vYC2dQXcEzEzK9dxNRFJb5V0P3A18P6qWXtkp7hulXRas9vhsbPMzMp1XIhExFURcRhwGnB+1awFEdEPnAF8UdKr8tYhaXkWOAPDw8MNtSPxfSJmZqU6LkQqslNfr5I0J/u8Ofv5EHAj8PqC766KiP6I6O/r62to++6JmJmV66gQkXSIJGXvjwZmAE9ImiVp92z6HOA44L5mtiXpgR074MUSjZmZjdXSq7MkrQMWA3MkDQIrgN0AIuJi4O3AeyS9ADwHvDO7Uus1wCWSdpAG3+ciorkhkghIeyO9STO3ZGY2dbU0RCJiacn8lcDKGtNvAV7XrHbV0lsJkdHY+d7MzF6qo05ndZIk+5dxXcTMLJ9DJIdDxMysnEMkR1J1OsvMzGpziORwT8TMrJxDJEfSk/ZEtrsnYmaWyyGSo3JZr3siZmb5HCI5XBMxMyvnEMnhmoiZWTmHSI6eHvdEzMzKOERyJK6JmJmVcojkSNwTMTMr5RDJ4auzzMzKOURyuCdiZlbOIZLDNREzs3IOkRw7eyI73BMxM8vjEMmx8z6R0fa2w8yskzlEclTuWPfYWWZm+RwiOXx1lplZuZaHiKQ1koYk3ZMz/1RJd0m6U9KApOOr5p0l6cHsdVYz2+maiJlZuXb0RNYCSwrm3wAcGRFHAe8HVgNImg2sAH4PWASskDSrWY10TcTMrFzLQyQiNgAjBfO3RkTlz/+9gMr7NwLXRcRIRDwJXEdxGO2SnaP4uidiZparI2sikt4q6X7gatLeCMA84JGqxQazabW+vzw7FTYwPDzcUBs8iq+ZWbmODJGIuCoiDgNOA87PJqvWojnfXxUR/RHR39fX11Ab/GRDM7NyHRkiFdmpr1dJmkPa8zioavZ8YHOztr3zjnXXRMzMcnVciEg6RJKy90cDM4AngO8DJ0malRXUT8qmNYVrImZm5XpbvUFJ64DFwBxJg6RXXO0GEBEXA28H3iPpBeA54J1ZoX1E0vnA7dmqzouI3AL9rnJNxMysXMtDJCKWlsxfCazMmbcGWNOMdo1VqYnscE3EzCxXx53O6hQexdfMrJxDJEfWEfHVWWZmBRwiOSTRm7gnYmZWxCFSIOmRn2xoZlbAIVIgcU/EzKyQQ6RA0iPfJ2JmVsAhUiDp8R3rZmZFHCIFkkRsd0/EzCyXQ6RAkrgnYmZWxCFSwDURM7NiDpECSY+vzjIzK+IQKZAkvk/EzKyIQ6SAeyJmZsUcIgXSS3zdEzEzy+MQKdCbyD0RM7MCDpECromYmRVziBRwTcTMrFhLQ0TSGklDku7Jmf8uSXdlr1skHVk172FJd0u6U9JAK9rrEDEzK9bqnshaYEnB/J8Db4iII4DzgVVj5p8QEUdFRH+T2vcSPp1lZlaspc9Yj4gNkhYWzL+l6uOtwPxmt6lI0uMnG5qZFenkmsgHgO9VfQ7gB5I2SlreigYkvjrLzKxQS3siEyXpBNIQOb5q8nERsVnS/sB1ku6PiA05318OLAdYsGBBw+1wTcTMrFjH9UQkHQGsBk6NiCcq0yNic/ZzCLgKWJS3johYFRH9EdHf19fXcFtcEzEzK9ZRISJpAfAd4N0R8bOq6XtJ2rvyHjgJqHmF12RyT8TMrFhLT2dJWgcsBuZIGgRWALsBRMTFwGeA/YCvSALYnl2JdQBwVTatF7giIq5tdns9FLyZWbFWX521tGT+MmBZjekPAUeO/0Zz+aFUZmbFOup0VqdJelwTMTMrssshIulwSW+XdOBkNKiT9CauiZiZFakrRCRdJOniqs9vA/4T+DZwn6TfneT2tZVrImZmxertiZwMVN9V/rfAd0nrFbeRFsq7Rvo8kXa3wsysc9UbInOBhwEkzQdeC3w2Iu4GvgR0V08kcU/EzKxIvSHyHDAze/8G4GmgMqLuVmDvSWpXR0jHzoIIB4mZWS31XuJ7B3C2pF8CZwPXRUSl9Hww8NhkNq7dkkQA7AjI3pqZWZV6Q+SvgWtJi+lPAX9WNe800rpI1+jN+mmjo2mvxMzMXqquEImI27OhSQ4DHoyIp6tmrwIenMzGtVtP1v1I6yLuipiZjVX3HesR8SywsXqapP0i4upJa1WHqPQ+fK+ImVlt9d4n8qeSPl71+XXZGFhDkgYkzZ30FrZR0pP1RHzXuplZTfWe6f8I6RVaFZ8nrY18FNgHOG+S2tURkiT96XtFzMxqq/d01gLgfgBJ+5Be5ntaRFwj6Qngs5Pcvrba2RPxvSJmZjXV2xNJgEqF4HjSR9bemH1+BNh/cprVGXorPRHXRMzMaqo3RB4E3py9Px24JSJ+k30+EBiZrIZ1AtdEzMyK1Xs66wLgcklnAbOAd1TNOwG4a7Ia1gkS90TMzArVe5/IFdnd6r8H3B4RG6pmPw6sn8zGtZtrImZmxeq+Dzsibo6IC8cECBGxIiKuKfqupDWShiTVfD66pHdJuit73SLpyKp5SyQ9IGmTpE/W2+5GVO4T2e6rs8zMaqo7RCTtKenDkr4t6QZJ35L0IUl7TuDra4ElBfN/DrwhIo4Azie9Cx5JCfBl0qHoDweWSjq83rbXqzJ2lmsiZma11Xuz4VzSQRi/BPQDe5IO/34RsFHSAUXfz3ovucX3iLglIp7MPt4KzM/eLwI2RcRDEfE88E3g1Hra3ghfnWVmVqzensj/JC2o/0FEHBwRx0bEwaSX++4LrJzEtn0A+F72fh7pJcQVg9m0pnJNxMysWCNPNvxURPzf6okRcQvwaV68/HeXSDqBNEQ+UZlUY7Hc3+ySlmfDsAwMDw833I6kahRfMzMbr94QmQlszpk3yIsPrGqYpCOA1cCpEfFE1boPqlpsfkE7iIhVEdEfEf19fX0NtyVJ3BMxMytSb4g8ALw7Z96ZZEOiNCobZv47wLsj4mdVs24HDpV0sKQZpDc6Nv1yYo/ia2ZWrJGbDS/LCuhXkD7JcC7pL/UTyQ8YACStAxYDc7LRf1cAuwFExMXAZ4D9gK9IAtie9Si2S/ow8H3SoVfWRMS9dba9bpWeyHZfnWVmVlO9Nxt+PbuU9zzSU04VjwMfjIgrSr6/tGT+MmBZzrxrgML7UCabayJmZsUaudlwFek4Wa8F/iD7OQ94WFKXDXuSPWPdNREzs5rqfrIhQETsAH5aPS0bGv61k9GoTuGaiJlZsbp7ItOJ71g3MyvmECngnoiZWTGHSIEXB2B0T8TMrJbSmoikV05wXXN3sS0dp3fnzYZtboiZWYeaSGF9EwVDjFTRBJebMlwTMTMrNpEQeV/TW9GhXBMxMytWGiIRcWkrGtKJHCJmZsVcWC/g01lmZsUcIgV8dZaZWTGHSAFJ9PT4dJaZWR6HSInEIWJmlsshUiJJ5JqImVkOh0gJ90TMzPI5REokPfLjcc3McjhESiSJH0plZpanpSEiaY2kIUn35Mw/TNKPJG2TdO6YeQ9LulvSnZIGWtNi6O1xTcTMLE+reyJrgSUF80eAc0if5V7LCRFxVET0T3bD8iSJayJmZnlaGiIRsYE0KPLmD0XE7cALrWtVMddEzMzyTaWaSAA/kLRR0vJWbTTpcU3EzCxPQ89Yb5PjImKzpP2B6yTdn/VsxslCZjnAggULdmmjSeKeiJlZninTE4mIzdnPIeAqYFHBsqsioj8i+vv6+nZpu0kPbHdPxMyspikRIpL2krR35T1wElDzCq/J5p6ImVm+lp7OkrQOWAzMkTQIrAB2A4iIiyXNBQaAlwE7JH0UOByYA1wlqdLmKyLi2la02TURM7N8LQ2RiFhaMv9XwPwas54GjmxKo0q4J2Jmlm9KnM5qJ4+dZWaWzyFSIvEd62ZmuRwiJTx2lplZPodIiV7XRMzMcjlESrgmYmaWzyFSwjURM7N8DpESHsXXzCyfQ6SER/E1M8vnECnhsbPMzPI5REr46iwzs3wOkRI9vk/EzCyXQ6SEayJmZvkcIiV8n4iZWT6HSIkk8X0iZmZ5HCIl/DwRM7N8DpESvYnYEbDDdREzs3EcIiWS7F/IGWJmNl5LQ0TSGklDkmo+H13SYZJ+JGmbpHPHzFsi6QFJmyR9sjUtTmsigOsiZmY1tLonshZYUjB/BDgHuKB6oqQE+DJwMukz15dKOrxJbXyJSk/EV2iZmY3X0hCJiA2kQZE3fygibgdeGDNrEbApIh6KiOeBbwKnNq+lL3KImJnlmyo1kXnAI1WfB7NpTVc5nbXdp7PMzMaZKiGiGtNyf6tLWi5pQNLA8PDwLm241zURM7NcUyVEBoGDqj7PBzbnLRwRqyKiPyL6+/r6dmnDPp1lZpZvqoTI7cChkg6WNAM4HVjfig07RMzM8vW2cmOS1gGLgTmSBoEVwG4AEXGxpLnAAPAyYIekjwKHR8TTkj4MfB9IgDURcW8r2uxLfM3M8rU0RCJiacn8X5Geqqo17xrgmma0q4h7ImZm+abK6ay26elxT8TMLI9DpERvkv50T8TMbDyHSAnXRMzM8jlESrgmYmaWzyFSInFNxMwsl0OkROKaiJlZLodIiUpPxGNnmZmN5xAp4auzzMzyOURKuCZiZpbPIVLCNREzs3wOkRI7eyJ+yLqZ2TgOkRI77xMZbW87zMw6kUOkxM4nG7onYmY2jkOkxM6aiHsiZmbjOERKuCZiZpbPIVLCNREzs3wOkRI7R/F1T8TMbByHSAmP4mtmlq+lISJpjaQhSffkzJekL0naJOkuSUdXzRuVdGf2Wt+qNvuOdTOzfK3uiawFlhTMPxk4NHstB75aNe+5iDgqe53SvCa+lMfOMjPL19IQiYgNwEjBIqcCl0XqVmBfSS9vTetq6+kRknsiZma1dFpNZB7wSNXnwWwawB6SBiTdKum0opVIWp4tOzA8PLzLjUp63BMxM6ul00JENaZVugALIqIfOAP4oqRX5a0kIlZFRH9E9Pf19e1yo5IeuSdiZlZDp4XIIHBQ1ef5wGaAiKj8fAi4EXh9qxqVJO6JmJnV0mkhsh54T3aV1jHAloh4TNIsSbsDSJoDHAfc14oGXX/bVp7bFlz5H89w+qcf5frbtrZis2ZmU0JvKzcmaR2wGJgjaRBYAewGEBEXA9cAbwI2Ab8B3pd99TXAJZJ2kAbf5yKi6SFy/W1bufCKJ4nsTNbQyCgXXvEkACcumtnszZuZdbyWhkhELC2ZH8DZNabfAryuWe3Ks3r9FrY9/9JayLbng9XrtzhEzMzovNNZHWV4pPaAWXnTzcymG4dIgb7ZSV3TzcymG4dIgWWn7MPuM8ZfdTw0Muoiu5kZoIjuvv+hv78/BgYGGv7+9bdtZfX6LQwVnMLaey8hxDPP7qBvdsKyU/ZxzcTMpixJG7P78sqXdYhMzOmffrQwSGqpDpeZE3jvADKzTlBPiLT06qyprJFi+jPPBpUb7ifyfmhklJWXj3DRt5+acPC08n3f7IRjXrsHt977W4ZHRjumTQ5es/ZxT2SCGumJWGuNPa3YaYHX6YE8Vdva6e1rZ1sb/SPLp7OqTFaIVG48HHvfiJlZJ9t9hvjYGbPqCpJ6QsRXZ03QiYtm8rEzZrG/L+81symkcoN0s7gmUocTF83cmeaVq7aqu6RPP+tRGs2s8zTzBmmHSIOqA6VarXCp51zmc78Ntrv0YmaTqJk3SDtEJlleuEzUroZQNxUFy96752dWbvcZYtkp+zRt/Q6RDrOrITTd5IVupwVepwfyVG5rp7dvKl6dVQ+HiE1pDl2z9vLVWWZm1jCHiJmZNcwhYmZmDXOImJlZwxwiZmbWsK4fO0vSMPCLBr8+B/j1JDZnKpiO+wzTc7+n4z7D9Nzvevf5FRHRN5EFuz5EdoWkgYkOQtYtpuM+w/Tc7+m4zzA997uZ++zTWWZm1jCHiJmZNcwhUmxVuxvQBtNxn2F67vd03GeYnvvdtH12TcTMzBrmnoiZmTXMIVKDpCWSHpC0SdIn292eZpF0kKQfSvqppHsl/Xk2fbak6yQ9mP2c1e62TjZJiaSfSPpu9vlgST/O9vlfJM1odxsnm6R9JV0p6f7smB/b7cda0n/P/tu+R9I6SXt047GWtEbSkKR7qqbVPLZKfSn7/XaXpKN3ZdsOkTEkJcCXgZOBw4Glkg5vb6uaZjvwsYh4DXAMcHa2r58EboiIQ4Ebss/d5s+Bn1Z9Xgl8IdvnJ4EPtKVVzfVPwLURcRhwJOn+d+2xljQPOAfoj4jfARLgdLrzWK8FloyZlndsTwYOzV7Lga/uyoYdIuMtAjZFxEMR8TzwTeDUNrepKSLisYi4I3v/DOkvlXmk+3tpttilwGntaWFzSJoPvBlYnX0W8EfAldki3bjPLwP+EPhngIh4PiKeosuPNenjLv6LpF5gT+AxuvBYR8QGYGTM5LxjeypwWaRuBfaV9PJGt+0QGW8e8EjV58FsWleTtBB4PfBj4ICIeAzSoAH2b1/LmuKLwF8ClUcj7gc8FRHbs8/deMxfCQwDX8tO462WtBddfKwj4lHgAuCXpOGxBdhI9x/rirxjO6m/4xwi46nGtK6+hE3STOBfgY9GxNPtbk8zSXoLMBQRG6sn11i02455L3A08NWIeD3wLF106qqWrAZwKnAwcCCwF+mpnLG67ViXmdT/3h0i4w0CB1V9ng9sblNbmk7SbqQB8o2I+E42+fFK9zb7OdSu9jXBccApkh4mPVX5R6Q9k32zUx7Qncd8EBiMiB9nn68kDZVuPtYnAj+PiOGIeAH4DvD7dP+xrsg7tpP6O84hMt7twKHZFRwzSAtx69vcpqbIagH/DPw0Ij5fNWs9cFb2/izg31vdtmaJiE9FxPyIWEh6bP8jIt4F/BD442yxrtpngIj4FfCIpFdnk/4bcB9dfKxJT2MdI2nP7L/1yj539bGuknds1wPvya7SOgbYUjnt1QjfbFiDpDeR/nWaAGsi4u/b3KSmkHQ88H+Au3mxPvBXpHWRbwELSP9HfEdEjC3aTXmSFgPnRsRbJL2StGcyG/gJcGZEbGtn+yabpKNILyaYATwEvI/0D8muPdaS/hZ4J+mViD8BlpGe/++qYy1pHbCYdLTex4EVwL9R49hmgXoR6dVcvwHeFxEDDW/bIWJmZo3y6SwzM2uYQ8TMzBrmEDEzs4Y5RMzMrGEOETMza5hDxKY8Se+VFJKeGjsKraTebN7fTOL23p+NjPq8pKcma72tImlh9m+yrN1tsanPIWLdZB/gE83cgKQDSZ8Sdwvp3e4nNnN7Zp3OIWLd5AfARyTNbeI2DiW9CfXSiLh5V27SMusGDhHrJn+X/fzrRr4s6dWSrspOiz0n6VZJS6rmrwVuzD7ekJ0SWluyzrdl6/lNtt5vS1owZpmHJX1d0p9mDwr6raQ7JJ1QY31nSvrPbJlfS7q81jDe2bruyPbjSUk3Sfr9MYslks6T9FjWtv+dDZNfvZ4zslF/t0raIuluSR8s2mebXhwi1k0eIx3OYbmkV9Tzxew01c2kD2v6MPAnwFPA1ZIqI7+eT/qQI4CzgWOzaXnr/DPSwS3vIx2r6YPA7wA3Sdp7zOJvAP6CNABPB7YB36sa6wpJy4HLSZ/78jbSUXjfmK1vZtVyF5Cecrsj248zgQ2kw19U+xRwCPB+0od0HQt8o2o9xwNfB24ifRbFO4D/Beybt882DUWEX35N6RfwXtKhrA8hHQ/pKdIxzyAdAj2AvylZxwWk4ysdUjUtAR4A7qiadmK2vsUl65tJ+vyKNWOmLwSeJx12vzLt4Wzagqppe5M+ZOjyqrY8DvxwzPqOz9pzTvb5EGAU+HxB2xZm37lpzPRzs+kHVn0eaffx9auzX+6JWFeJdPDAC0lHKX112fJV/hC4NSI2Va1rFFgHHJU9GbAexwIvA76RXSHWmw0/Pgjcn22v2q0R8cuqbT8DXJ2tB+DVpA8V+kb1lyLiZuAXpD0ZSEOuh7QnUubqMZ/vzn5Weiy3A7OyU21vkeQeiI3jELFu9AXSv+LPq+M7s0lPh431K9KH+MyqMa9I5Sly1wMvjHm9jvRpitUer7GOx3nxiXOzs595bazMr6x3cAJtHDtab2Uk2z0AIuIm0lNYBwFXAcOSrpd0xATWbdNEb/kiZlNLRGyV9FnSHsk/TvBrI0Ctq7rmkp7iqXd49Ceyn+8F7q0x/5kxnw+oscwBwKNV7au0Z6y5QOUqsV9nP+eRnorbJRFxJXBlVnNZDKwErpU0PyJ2FH7ZpgX3RKxbfYX0F/DflS2YuYn0AUYLKxMkJaTPovhJdnqpHreQBsUhETFQ4zX2F/wxknY+bS4rvL8Z+FE26QHSnsnp1V/Krrh6RdZ+SHs+O4Dldba3UERsjYjvApcAL2d8T8qmKfdErCtFxDZJ5zGx2gCkp8DeC1wnaQXwNPAh4L+S/jKvd/tPS/o48GVJfcD3SAvt80jrFzdGxBVVX3kc+EF2Z/020psm9yK7+isiRiV9BrhE0tdJr5qaB/w98CDwtWy5/yfpC8BfZEG0nrTQvgi4PyL+ZaL7kP37HUD6JMDNpI9RPQe4MyKG6/03se7kELFu9jXg46Q3CBaKiM3ZJa0rga8CuwN3Am+OiGsb2XhEXCLpkawNZwC7kfaONmTrrnYT6T0o/0D6y/o+4OSI+FnV+lZJ+k22vn8HtgLXAH8ZEVurljtX0ibSEDwLeBa4i/RmzHr8mDQ0vkBacxnK1vE/6lyPdTE/2dCszSQ9DNwcEWe2uy1m9XJNxMzMGuYQMTOzhvl0lpmZNcw9ETMza5hDxMzMGuYQMTOzhjlEzMysYQ4RMzNrmEPEzMwa9v8BRlgYfPL9OuoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(Loss,color='RoyalBlue',marker='o')\n",
    "plt.xlabel('N of epochs',fontsize=16)\n",
    "plt.ylabel('Loss',fontsize=16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-06T08:02:10.492151Z",
     "start_time": "2019-07-06T08:02:10.213737Z"
    },
    "slideshow": {
     "slide_type": "slide"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjgAAAEYCAYAAABRMYxdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XlcVOX+B/DPAA67orLIoiwugKCiwsU1N9zwqpjmkuYWmUs306vm1bx6yz0rzYVCy7XS1NwVRRFBxQTRTA2XEBUFAQXZFASe3x/zY2pkUJCBMwyf9+s1L5szM+d8hmm+fDnnec6RCSEEiIiIiHSIntQBiIiIiDSNDQ4RERHpHDY4REREpHPY4BAREZHOYYNDREREOocNDhEREemcGtHg/PTTT5DJZIiIiFBZ/vDhQ8hkMtjY2JR4zdq1ayGTyXDlyhUAwJUrV/D++++jbdu2kMvlkMlk5c6xe/du2NjYIDc39/XeSDmMHTsWTk5OyvtJSUkwMTHB+fPnK33b5eHk5ISxY8eW+nhSUhJkMhk+/fTTEo+5u7tDJpPh2rVrKsuvXr0KmUyGNWvWAABOnDiBUaNGoXHjxjA2Nkbjxo0xadIkpKSklDnnhQsXYGJigvv375f5NVXtq6++QsuWLVFUVCR1FKohAgMDIZPJMH36dKmjaI0Xay9Jp0Y0OF26dAGAEg1OREQETExMkJKSgri4uBKP1a9fHx4eHgAUv+AOHz6MRo0awdvbu9wZCgoKMGfOHMycORMmJiav+U5en62tLd577z3MnDmzyrddEba2tmjSpEmJzy41NRVxcXEwMTFR+7kCwBtvvAEA+Oabb/Do0SN88sknCAkJwX/+8x/s378f7dq1Q3Z2dplyzJw5E+PHj4e9vb0G3lXlmDhxIlJSUrB582apo1AN8PTpU+zcuRMA8MMPP6CgoEDiRESqakSDY2dnBxcXF7W/CLt3747GjRuXeCwyMhKdO3dW7ql55513cO/ePezZswfdu3cvd4Z9+/YhISEB48ePf+nzCgsLK61QvP/++4iIiNC6vTiv8sYbbyAqKgrPnz9XLouIiECdOnUwdOhQtZ9rvXr10KJFCwDAunXrcOTIEYwbNw5dunRBYGAgfvrpJ9y+fRs///zzK7cfGxuLkydPYtKkSRp7T3l5eRpbVzFjY2OMHj0aK1as0Pi6iV60Z88eZGZmwt/fHykpKQgJCanS7T9//hw8Ty29TI1ocADFXpyoqCiV5iEiIgKdO3dGp06dVH5J3rx5E0lJSco9AACgp1exH9WGDRvQp08f1KtXT2W5TCbD3LlzsXTpUjg7O0Mul+P3338HAKSlpWHSpEmwt7eHoaEh3NzcEBwcXGLdJ06cQJs2bWBkZITGjRvj22+/VZuhefPmaNGiBTZs2PDKvMeOHYO/vz9sbW1hYmICT09PfPHFFygsLFR5npOTE0aNGoXt27fD3d0dpqam8Pb2xunTp0usc9WqVXBycoKRkRG8vb0RGRn5yhyA4rPLzc3FhQsXlMsiIiLQoUMHdOnSRW1z2qlTJ2VzamVlVWKdPj4+AFCmQ07r169Hy5YtlXvziuXm5mLSpEmoX78+zM3NMWjQIJw9exYymQybNm1SPm/s2LFwcHBAVFQUOnToAGNjY8yaNQsAsH37dnTv3h1WVlYwMzND69atS+yBadGiBQYNGlQiV3h4OGQyGY4ePapcNnz4cFy7dg1nz5595fsiqojNmzejbt262LRpE4yNjbFlyxblYz///DNkMhkuX75c4nV9+/aFl5eX8n5BQQGWLFkCNzc3GBoaws7ODv/+97/x7Nkz5XMSEhIgk8mwbt06zJo1C3Z2djA0NERGRgZSU1Px/vvvo1mzZjAxMUHDhg3x9ttvq/1u//TTT3Bzc4ORkRFatGiB/fv3o2vXrujatavK8zRde0kioobYuHGjACB+/fVXIYQQ6enpQk9PT5w9e1Zs2LBBNGzYUPncDRs2CAAiJiZG7brmzp0ryvOje/bsmTAyMhKff/55iccACDs7O9GpUyexa9cuceTIEZGcnCyePHkimjVrJho2bCiCg4NFaGiomDFjhtDT0xNff/218vXXrl0TcrlcdOjQQezZs0ds375duLm5CQcHB+Ho6Fhie1OmTFF5r6UJCgoSK1asEIcPHxZhYWFi+fLlwszMTHz88ccqz3N0dBSNGjUS3t7eYufOneLAgQPCy8tL1KlTR6SnpyufV/wzHTt2rDhy5IhYvXq1sLe3F7Vr1xZjxox5aZbbt28LAGLZsmXKZV5eXmLx4sXi1q1bAoD4888/hRBCeX/FihUvXeeRI0cEALFz585X/iycnJzElClTSiwfOXKkkMvlYvHixeLYsWPi448/Fk5OTgKA2Lhxo/J5Y8aMEWZmZqJRo0bi66+/FidPnhTnzp0TQgixaNEisXbtWnH06FERGhoq5s2bJwwMDERQUJDy9WvXrhUGBgbi/v37KtsfPny4cHZ2FkVFRcplhYWFonbt2mLevHmvfF9Er+v+/ftCT09PTJw4UQghxIgRI4ShoaF4/PixEEKIp0+fijp16oiZM2eqvC45OVkYGBiofD+HDRsmTExMxP/+9z8RGhoqvv76a1GnTh3x5ptvKp9TXAPs7OzEwIEDxYEDB8TevXtFbm6uiIuLEx9++KHYtWuXOHXqlPjpp5+Et7e3cHR0FE+fPlWu49ixY0Imk4mBAweKgwcPik2bNglnZ2dha2srunTponxeZdVeqno1psH5888/BQBlk7F//35hbGws8vLyxPXr1wUAcfv2bSGEEKNHjxa1a9cWBQUFatdV3gbn3LlzAoA4duxYiccACFtbW5Gbm6uy/NNPPxWGhobixo0bKssDAwNF/fr1xfPnz4UQQrz99tuifv36Ijs7W/mcu3fvilq1aqn9khU3Gi/+snyZoqIi8fz5c7Fw4UJhYWEhCgsLlY85OjoKCwsLZWETQojo6GgBQPzwww9CCMUvXQcHB9G7d2+V9W7fvl0AeGWDI4QQjRo1Ev369RNCCJGRkSH09PREZGSkEEIIW1tbZUPx/fffCwAiOjq61HVlZmYKV1dX4e7urvw5liY5OVkAEMHBwSrL4+LihEwmU2m6hBDiX//6l9oGB4DYu3fvS7dVWFgonj9/LgIDA0XLli1V8pqbm4tPP/1UuSw1NVXI5XKxZMmSEuvp1KmT6Nmz50u3RVQRS5cuFQDE2bNnhRBChISECAAqjXlgYKCwt7dXqRdfffWV0NfXFw8ePBBCCBERESEAiM2bN6usf9u2bQKAuHjxohDirwandevWKg29OgUFBeLu3bsCgPjll1+Uy9u3by88PDxUXn/hwgUBQKXBqazaS1WvxhyicnFxgYODg/JwRkREBHx9fSGXy9GsWTNYW1urPNaxY0fo6+trZNsPHjwAoP5QCQD06dMHxsbGKstCQkLg6+sLZ2dnFBQUKG+9e/fGo0ePlDOHoqKi4O/vD1NTU+VrGzZsiI4dO6rdVnGG4kylSUpKwvvvvw9HR0fI5XLUqlULn3zyCTIyMkrMPmrfvj3q1q2rvF889uXu3bsAgMTERCQmJmLo0KEqrxs8eDAMDAxemqPYG2+8gTNnzqCoqAinT59GrVq1lIeZ/n6IMSIiAubm5mjdurXa9RQUFGDEiBG4f/8+tm/f/srtl/bZ/frrrxBC4K233lJZPmTIELXrMTAwwD//+c8Sy2/evIkRI0bA3t4etWrVQq1atbBhwwZcv35d+Rxzc3OMGjUKGzZsUM6Q2rhxI4QQGDduXIl1WllZvfLzJaqILVu2oGnTpmjfvj0AwM/PD3Z2diqHqd555x3cv38fYWFhymVbt26Fn58fbG1tASjqnFwux+DBg1XqXK9evQCUnBgSEBCgdgZrUFAQWrVqBTMzMxgYGKBRo0YAoPweFRYWIiYmBoMHD1Z5fZs2beDs7KyyrsqqvVT1akyDAyh+SZ4+fRpCCOX4m2LFvyQTExORkJCgMv6mooqPJRsaGqp9vPjL/ncpKSmIiIhQ/tIrvhX/Qn306BEARSOibpq7umUAlI3U06dPS81bVFSEAQMG4ODBg/jkk08QFhaG6OhozJ07V+X9FHtxXFHx+yx+XlJSktpMBgYGqF+/fqk5/u6NN95ARkYGLl++jIiICPzjH/9Qbqdz584qDU6HDh3UNqdFRUUYM2YMjh8/jr1796Jly5av3G5pn13xe7K2tlZZXtrP3draukSm7Oxs9OzZE7/99huWLl2KyMhIREdHY/z48SUGIU+ePBl3797F4cOHIYRAcHAwBg0apHZ7xsbGL/18iSoiOjoa165dw5tvvomMjAxkZGQgKysLb775JqKionDjxg0Aiu+lk5MTtm7dCgD4448/EBsbi3feeUe5rpSUFOTn58PMzEylzhV/r4rrXDF1tXL16tWYPHky/Pz88Msvv+D8+fM4d+4cgL++v2lpaXj+/HmJ7ytQ8jtbWbWXql7Z/nzWEW+88QZ+/PFHnDt3DrGxsVi4cKHysc6dO2PdunU4deoUgL+mlmtC8S/x9PR0tY+r+4ukfv36sLa2xqpVq9S+xtXVFYDiC//w4cMSj6tbBgCPHz8GAFhaWpaa988//0RMTAy2bt2KUaNGKZcfOHCg1Ne8THFRejFTQUFBiQJWmuKGMyIiQjn7rVjnzp3x4YcfIjo6GvHx8QgMDFS7jokTJ2LHjh3YtWsXevToUabtlvbZFb+nlJQUlb8AS/u5q/uMo6KicOfOHeWg6GLqZtF5enqic+fO+Pbbb2FkZIRbt26VOqDx8ePHL/18iSqieBD8smXLsGzZshKPb9myBQsXLoRMJsOoUaOwcuVKBAUFYevWrTAzM1MZMF+/fn0YGRmVOuHAzs5O5b6679H27dvRo0cPfPHFF8plt2/fVnmOpaUlatWqpfbcVw8fPlTu8SnOVBm1l6pejdqDU9y0LF26FEII5e5VQLEH5+bNm/j5559hYmLyWue6KY2bmxsAID4+vsyv6dOnD+Li4pTn3XnxZm5uDkBxeOjw4cPIyclRvvbevXs4c+aM2vXevn0bcrm8xG7Zvys+EWGtWrWUy54/f44ffvihzPn/zsHBAQ0bNiwxJXv37t1lnhLv6uqKBg0aICQkBBcuXFDZ+9ayZUvUrl1b2bCq2/v273//Gxs2bMDGjRsREBBQ5uzFs75e/Ox8fX0hk8mU5wEp9uL9l1H3c05PT8e+ffvUPn/y5Mk4cuQIFixYgGbNmpV6uoLbt28rizCRJuXn52P79u3w9fXFyZMnS9y8vLywdetW5fTtd955B9nZ2fjll1/www8/YPDgwSrnAevTpw+ePXuGJ0+eqK1zLzY46uTm5qp8hwDFIdy/09fXh7e3N3bv3q0ytfzChQslmqHKqr0kAQnH/0jC2tpayGQy4e3trbK8oKBAmJmZCZlMJrp3717idTk5OWLnzp1i586dYvDgwcoZODt37nzpgNZijo6O4l//+leJ5QDE3LlzSyzPyMgQbm5uolmzZiIoKEiEhYWJAwcOiM8//1wMGDBA+bxr166JWrVqKUfy79ix46Uj+QMCAkTHjh1fmjUvL084OjqKxo0bi507d4q9e/eKLl26iMaNG6sMxi5+XyNHjlT7vubPn6+8//dZVCEhIWLNmjVlnkVV7K233hIymUzo6emJJ0+eqDzWp08fIZPJlAPH/654QOT48eNFVFSUyu3WrVuv3G6XLl1E//79SywvnkW1ZMkScezYMTF79mzh6OhYYtDkmDFjhL29fYnXp6SkiNq1a4u2bduKgwcPih07dogWLVoof84vys/PFzY2NgKA+OKLL9RmTU9PFzKZTKxfv/6V74uovHbv3i0AiE2bNql9PCgoSAAQYWFhymW+vr7C3t5eABDHjx8v8ZoRI0YICwsL8emnn4qQkBBx7NgxERwcLAICAsT169eFEH8NMlb3//Xs2bOFTCYTixYtEqGhoeI///mPaNq0aYkadOzYMQFADBw4UBw6dEhs3rxZODs7iwYNGohu3bopn1dZtZeqXo1rcIYMGSIAiGnTppV4rGfPngKAWLBgQYnHir9g6m5l+QU9a9Ys4ezsXGJ5aQ2OEEI8fvxYfPTRR8LJyUnUqlVLWFlZiU6dOomvvvpK5XmhoaHCy8tLyOVy4ezsLL755hsxZsyYEl+y3NxcYW5uLlavXv3KvBcvXhQdO3YUxsbGwt7eXsybN0+sX7/+tRscIYRYuXKlaNSokTA0NBRt27YVkZGRwtHRscwNzpo1a5QzKV60aNEiAUB07dq1xGNdunSp0Ge3bt06YWpqqjJbQghF0ztx4kRRt25dYWpqKvr37y8OHjxYYsZUaQ2OEEKcOHFCeHl5CSMjI+Hi4iJWrVol5s+fX+osvQkTJghDQ0ORlpam9vFt27a99HGiihgwYIAwNzcXOTk5ah/PyMgQxsbGKt+r4u/tizOqihUWFoqVK1eKli1bCkNDQ1G7dm3RsmVLMXPmTJGRkSGEeHmDk5ubKyZOnCgsLS2FmZmZ6Nevn4iPj1dbg3744QfRrFkzIZfLRfPmzcUvv/wivLy8REBAgMrzNF17SRoyIXgqyKrw559/wtXVFeHh4SrjLarSjh07EBgYiHv37sHCwkKSDNVRZmYmHBwcsG7dOpUxSep8/vnn+Pjjj5GQkKByXF8TCgoK0KRJE3Tu3Fk5cPNFffv2haWlZamPE9FfEhMT0aRJE8ydOxfz5s2TOg5pGBucKvTee+8hKSkJBw8elGT7bdq0QUBAAP773/9Ksv3qbNGiRdixYwd+++035UDHgwcP4sqVK/Dy8oKenh4iIyOxYsUKDBw4ENu3b9fYtjMzM3HlyhX8+OOPCAoKwoULF1TOBFvs0qVLaNeuHa5cuYImTZpobPtEuuDp06eYPn06/Pz8YGlpifj4eCxfvhwPHz7E1atX1c7QouqtRs2iktpnn32Gb7/9Frm5uVV+wc3k5GQMHDgQM2bMqNLt6orp06ejsLAQSUlJyoGP5ubm2Lt3L5YuXYqcnBzY29vjww8/xP/+9z+Nbjs2NhbdunVTzuxQ19wAis9448aNbG6I1NDX10dycjI++OADPHr0CKampujcuTN27tzJ5kZHcQ8OERER6ZwaNU2ciIiIagZJDlFZWlrCyclJik0TUSVKSEhAWlqa1DGUWGuIdE9Z64wkDY6TkxNiYmKk2DQRVSJNniBTE1hriHRPWesMD1ERERGRzmGDQ0RERDqHDQ4RERHpHDY4REREpHPY4BAREZHOYYNDREREOocNDhEREekcNjhERESkc9jgEBERkc5hg0NEREQ6hw0OERER6Rw2OERERKRzNNLgZGRkYMiQIXBzc4O7uzuioqI0sVoiIhWsNURUVhq5mvjUqVPRp08f7Nq1C/n5+cjNzdXEaomIVLDWEFFZVbjByczMREREBDZt2gQAkMvlkMvlFV0tEZEK1hoiKo8KH6KKj4+HlZUVxo0bh9atWyMwMBA5OTklnhccHAxvb294e3sjNTW1opslohqGtYaIyqPCDU5BQQFiY2MxadIkXLx4Eaampli6dGmJ502YMAExMTGIiYmBlZVVRTdLRDUMaw0RlUeFGxwHBwc4ODjA19cXADBkyBDExsZWOBgR0d+x1hBReVS4wWnQoAEaNmyI69evAwBOnDiB5s2bVzgYEdHfsdYQUXloZBbV6tWrMXLkSOTn58PFxQUbN27UxGqJiFSw1hBRWWmkwfHy8kJMTIwmVkVEVCrWGiIqK57JmIiIiHQOGxwiIiLSOWxwiIiISOewwSEiIiKdwwaHiIiIdA4bHCIiItI5bHCIiIhI57DBISIiIp2jkRP9EWlaXl4eTp48ibi4OOTk5KBBgwbo06cP7O3tpY5GRETVAPfgkFY6cOAA/vzzTwQEBGDSpElo3Lgxtm7diszMTKmjERFRNcAGh7TO8+fPce3aNfTo0QNOTk6oV68eunbtinr16vE0/UREVCZscEjrFBUVQQgBAwPVI6gGBga4e/euRKmIiKg6YYNDWsfQ0BAODg6IjIxEZmYmioqKcPnyZSQmJiI7O1vqeEREVA1wkDFppUGDBmH//v346quvIJPJYGtrC09PTyQlJUkdjYiIqgE2ODVAUVERwsPD8fvvvyMrKwvm5uZo0aIFunbtCj097dyJV69ePYwdOxb5+fnIy8uDubk5du3ahbp160odjYiIqgE2ODXAmTNnEB0djYCAANjY2ODhw4fYu3cv9PX10aVLF6njvZRcLodcLsfTp09x69Yt9OzZU+pIRERUDbDBqQHu3bsHV1dXuLq6AgAsLCzg6uqK+/fvS5ysdLdu3YIQApaWlnj8+DFCQ0NhaWkJLy8vqaMREVE1wAanBmjUqBGio6ORlpYGS0tLpKam4vbt2+jUqZPU0UqVl5eHEydOIDMzE8bGxnB3d0f37t2hr68vdTQiqmay8rIw7+Q87Inbg5ScFLRu0Bqr+qyCj72P1NGoErHBqQE6duyIvLw8rF27Fnp6eigqKkLnzp3h46O9X24PDw94eHhIHYOIdEDggUBcfngZmwM2w6G2A7Zd3ga/rX64Nvka7Gvz7Oi6ig1ODXD16lVcvnwZgwcPhpWVFZKTkxESEgILCwu0adNG6nhERJXm6fOn2H1tN3YP3Y2uTl0BAAu6LsCBGwcQFBOEhd0XShuQKg0bnBogNDQU7du3h6enJwDAxsYGT548wenTp9ngEJFOKygqQKEohJGBkcpyYwNjnL57WqJUVBW0c44wadTz589LTAeXyWQQQkiUiIioapgbmqO9Q3ssjFyI+5n3UVhUiG2XtyEqMQpJ2Tyvli5jg1MDNGvWDKdPn8aNGzeQkZGBP/74A+fOnYObm5vU0YiIKt3WQVuhJ9ODw1cOMFxoiK9//RojPEdAX8ZJC7qMh6hqgL59++LkyZM4fPgwcnJyYGZmhjZt2mj9OXCIiDShcb3GODX2FHLyc5CZlwlbc1sM2zUMznWdpY5GlUhjDU5hYSG8vb1hb2+PgwcPamq1pAGGhobo06cP+vTpI3UUogpjraHXZSo3hancFOlP03H01lEs77lc6khUiTTW4KxatQru7u7IzMzU1CqJiEpgraHyOnrrKIpEEdws3XDr8S3MDJ0JV0tXjPMaJ3U0qkQaGYOTmJiIQ4cOITAwUBOrI12TlQV89BHg6AgYGwMdOgDR0VKnomqItYZex5O8J/jgyAdwW+uG0XtHo1OjTjg26hhq6deSOhpVIo3swfnoo4+wfPlyZGVllfqc4OBgBAcHAwBSU1M1sVmqLgIDgcuXgc2bAQcHYNs2wM8PuHYNsOdJtqjsWGvodQz1GIqhHkOljkFVrMJ7cA4ePAhra2u0bdv2pc+bMGECYmJiEBMTAysrq4pulqqLp0+B3buBpUuBrl2BJk2ABQsU/wYFSZ2OqhHWGiIqjwo3OGfOnMH+/fvh5OSE4cOHIywsDKNGjdJENtIFBQVAYSFgpHqSLRgbA6d5ki0qO9YaIiqPCjc4S5YsQWJiIhISErB9+3Z0794d27Zt00Q20gXm5kD79sDChcD9+4pmZ9s2ICoKSOJJtqjsWGuIqDx4oj+qfFu3Anp6ivE3hobA118DI0YAvDI4ERFVEo2e6K9r167o2rWrJldZo6xcuRJPnjwpsbxp06Z4++23JUikIY0bA6dOATk5QGYmYGsLDBsGOPMkW/R6WGuI6FV4JmMt8t5776lcHyorKwvBwcFo3ry5hKk0yNRUcUtPB44eBZbzJFtERFQ52OBoEVNTU5X7sbGxMDQ0hIeHh0SJNOToUaCoCHBzA27dAmbOBFxdgXE8yRYREVUOjsHRUkIIXLx4ES1btkStWtX8ZFRPngAffKBocEaPBjp1Ao4dA6r7+yIiIq3FPThaKj4+HhkZGWjTpo3UUSpu6FDFjYiIqIpwD46Wio2NhZ2dHRo0aCB1FCIiomqHDY4WysnJQVxcnG7svSEiKkVhUSHmhc2D8ypnGC00gvMqZ3wS9gkKigqkjkY6gIeotNClS5dgYGAAT09PqaMQEVWaZWeWYW30WmwO2IwWNi1w+eFljNk7Bob6hpjXZZ7U8aiaY4OjZYQQiI2NhYeHBwwNDaWOQ0RUac7eO4v+rv3R37U/AMDJwgkDXAfg1/u/SpyMdAEPUWmZhIQEPH78+JUXFCQiqu46NeqEk7dPIi4tDgBwLfUawm6Hwb+pv8TJSBdwD46WcXZ2xvz586WOQURU6T7u+DGy8rLQfG1z6Ovpo6CoAHM7z8Vkn8lSRyMdwAaHiIgksePqDmy5vAU/Dv4RHlYeuJR8CVNDpsLZwhnvtnlX6nhUzbHBISIiScwMnYkZ7WdguOdwAEALmxa48+QOlpxewgaHKoxjcIiISBK5z3Ohr6evskxfpo8iUSRRItIl3INDRESS6N+sP5aeXgpnC2d4WHvgYtJFfHnuS4xuOVrqaKQD2OAQEZEkVvddjXkn52Hy4clIyUmBrZkt3mvzHv7b5b9SRyMdwAaHiIgkYW5ojpV9VmJln5VSRyEdxDE4REREpHPY4BAREZHO4SEqIiIqE6eVTrjz5E6J5f5N/XHo7UMSJCIqHRscIiIqk+j3olEoCpX3k7KS0Da4LYY2HyphKiL12OAQEVGZWJlaqdz/LvY71Dasjbc83pIoEVHpOAaHiIjKTQiB7y5+h1EtR8GklonUcYhKYINDRETlFhofitsZtxHYJlDqKERqscEhIqJyWx+7Hj52PvBq4CV1FCK1Ktzg3Lt3D926dYO7uzs8PDywatUqTeQiIlLBWqM9UnJSsC9uH95r857UUYhKVeFBxgYGBvjiiy/Qpk0bZGVloW3btujZsyeaN2+uiXxERABYa7TJpkubYGhgqLwKOJE2qvAeHFtbW7Rp0wYAYG5uDnd3d9y/f7/CwYiI/o61RjsIIbAhdgOGewyHuaG51HGISqXRMTgJCQm4ePEifH19SzwWHBwMb29veHt7IzU1VZObJaIahrVGOuEJ4bj5+Cbea8vDU6TdZEIIoYkVZWdno0uXLpg7dy7efPPNlz7X29sbMTExmtgsEWmRqvhus9YQ1Wxl/V5rZA/O8+fPMXjwYIwcOfKVBYeI6HWx1hAGzgBZAAAep0lEQVRRWVW4wRFC4N1334W7uzumT5+uiUxU3RQWAvPmAc7OgJGR4t9PPgEKCqRORjqEtYaIyqPCDc6ZM2ewdetWhIWFwcvLC15eXjh8+LAmslF1sWwZsHYt8PXXQFwcsGqV4v6SJVInIx3CWkNE5VHhaeKdOnWChobxUHV19izQv7/iBgBOTsCAAcCvv0oai3QLaw0RlQfPZEwV16kTcPKkYu8NAFy7BoSFAf7+0uYiIqIai1cTp4r7+GMgKwto3hzQ11eMvZk7F5g8WepkRERUQ7HBoYrbsQPYsgX48UfAwwO4dAmYOlUx2Pjdd6VOR0RENRAbHKq4mTOBGTOA4f9/2vYWLYA7dxSDjNngEBGRBNjgVKGsrCycOHECN2/eRF5eHurWrYt+/frByclJ6mgVk5urODT1d/r6QFGRNHmIiKjGY4NTRZ49e4bvv/8ejRo1wttvvw0TExOkp6fD1NRU2mBOToq9LS/y9wcOHSrbOvr3B5YuVRyS8vAALl4EvvwSGD1ao1GJiIjKig1OFTlz5gzMzc0xaNAg5bK6detKmOj/RUcrTtRXLCkJaNsWGDq07OtYvVpxor/Jk4GUFMDWFnjvPeC//9V8XiIiojJgg1NF4uLi0KRJE+zatQu3b9+Gubk52rRpAx8fH8hkMumCWVmp3v/uO6B2beCtt8q+DnNzYOVKxY2IiEgL8Dw4VSQ9PR3R0dGwsLDAqFGj4Ovri+PHjyM6OlrqaH8RQtHgjBoFmJhInYaIiOi1cQ9OFRFCwM7ODn5+fgAAW1tbPH78GNHR0fjHP/4hcbr/FxoK3L4NBAZKnaTGi46OxtmzZ5GVlQVra2v07t0bjo6OUsciIqo2uAenipibm8PqhcNBlpaWePLkiUSJ1Fi/HvDxAby8pE5So125cgUhISHo1KkT3n//fTg4OOCHH37Qrv9XSKckZSVhzN4xsPrcCkYLjdB8bXOcSjgldSyiCmGDU0UaNmyIR48eqSx79OgR6tSpI1GiF6SkAPv2KQYHk6TOnTuHVq1aoW3btrCysoK/vz/Mzc2163Am6YyMZxno+H1HCCFw6O1D+GPKH1jddzWsTa2ljkZUITxEVUXatWuH77//HhEREfD09ERSUhLOnz+P7t27Sx1NYdMmwNDwr5P1kSQKCwvx4MEDtG/fXmW5i4sLEhMTJUpFumz5meWwNbfFlkFblMuc6zpLmIhIM7gHp4rY29tj2LBhuHbtGtatW4ewsDB069YNPj4+UkdTDC7esEHR3JibS52mRsvNzYUQAmZmZirLzczMkJ2dLVEq0mV74/bC194Xw3YNg/Xn1vD6xgtrzq/hldup2uMenCrUrFkzNGvWTOoYJYWHAzdvAtu2SZ2ESsFfNlRZ4tPjsS56Haa1m4bZo2bjUvIl/OvIvwAAH/zjA4nTEb0+NjgEdOum2ItDkjMxMYFMJiuxtyYnJ6fEXh0iTSgSRfC288YSvyUAgNa2rXHz8U2sjV7LBqeaWxe9Dp+f/RxJWUnwsPbAyt4r0dmxs9SxqgwPUemapCRgzBjFCfyMjIDmzYFTnA1RXejr68POzg7x8fEqy+Pj4+Hg4CBRKtJltua2aG7VXGWZu6U77j65K1Ei0oQdV3ZgashUzOk0Bxffv4gODh3Q94e+NepzZYOjSzIygI4dFXtjDh0C/vhDcRkFa86GqE7atWuHS5cuITY2FqmpqThy5AiysrLg7e0tdTTSQR0bdsT1R9dVlt14dAOOdXjepersy3NfYmyrsXiv7Xtwt3LHav/VsDW3RVB0kNTRqgwPUemS5csV14Ha8tdsCDhzNkR14+npiadPnyIiIgLZ2dmwtrbGyJEjYWFhIXU00kHT2k1Dh+87YFHEIgzzHIaLSRfx9fmvsbj7Yqmj0WvKL8zHhQcXMKP9DJXlvVx64WziWYlSVT02OLpk716gTx9g2DDg5EnAzk5xVuIpUwApr3dF5ebj46MdM+xI5/nY+2DvsL2YEzYHn0V8hkZ1GuGzbp9hss9kqaPRa0rLTUOhKISNmY3KchszGxy/fVyiVFWPDY4uiY8H1q0Dpk0DZs8GLl0C/qWYDYEPOFiQiNTr16wf+jXrJ3UM0jAZVP+wFUKUWKbL2ODokqIiwNsbWKKYDYHWrRXTv9euZYNDRFRDWJpYQl+mj+TsZJXlKTkpJfbq6DIOMtYltraKWVN/5+4O3K05o+a1wrp1irFPRkZA27ZAZKTUiYioBpHry9HWri1C40NVlofGh6KDQweJUlU9Nji6pGNH4LrqbAjcuAHwKtRVZ8cOYOpUYM4c4OJFoEMHoG9fNplEVKWmt5uOTZc2YUPsBvyR+gemHpmKB1kPMNF7otTRqoxGGpyQkBC4urqiSZMmWLp0qSZWSa9j2jTg3Dlg0SLg1i1g507g668Vg4ypanz5JTB2rOKipe7uimn6trZAUM2ZmlmZWGuIymaY5zCs7LMSCyMWwutbL5y+dxqHRx6Go0XN+YO3wmNwCgsLMWXKFISGhsLBwQE+Pj4YMGAAmr94qIQqn4+PYibVnDnAZ58BjRop/p3M2RBVIj8fuHABmKE6NRO9egFna87UzMrCWkNUPpN9Jtfo2XAV3oNz/vx5NGnSBC4uLpDL5Rg+fDj27duniWz0Ovr1A377DXj2THF46sMPOUW8qqSlAYWFgM0Lg/hsbIDkZPWvoTJjrSGi8qhwg3P//n00bNhQed/BwQH3798v8bzg4GB4e3vD29sbqampFd0skUJEBDBgAGBvr2jkNm2SOlHJhlIINpkawFpDROVR4QZH3VWOZWqK+YQJExATE4OYmBhYWVlVdLNECtnZgKcnsGoVYGwsbRZLS0Bfv+TempSUknt1qNxYa4ioPCrc4Dg4OODevXvK+4mJibCzs6voaonKxt8fWLwYGDIE0JN4UqBcrpgWHqo6NROhoYrZVFQhrDVEVB4V/o3g4+ODmzdv4vbt28jPz8f27dsxYMAATWQjqn6mT1ccJtuwQXGx06lTgQcPgIk1Z2pmZWGtIaLyqPAsKgMDA6xZswa9e/dGYWEhxo8fDw8PD01ko8qwYAHwv/+pLuMgWM0ZNgx49AhYuBBISlIcPjt8mOci0gDWGiIqD41cqsHf3x/+/v6aWBVVBVdXIDz8r/v6+pJF0UmTJ3NqfiVhrSGisuK1qGoiAwOgQQOpUxAREVUaXqqhJoqPV0yrdnYGhg9X3CciItIh3INT0/j6KgbBurkppi8vXKiY4XP1KlC/vtTpyi87W3FZCkBxNfW7d4FLl4B69RRnciYijVgQvgD/O6U6fs/G1AbJMzh+j7QTG5yapm9f1fvt2gEuLsDmzYoZQNVNTAzQrdtf9+fPV9zGjNGOk/4R6RDX+q4IHxuuvK8v4/g90l5scGo6MzPAwwO4eVPqJK+na1fFmYKJqNIZ6BmggRnH71H1wAZHy4SHh+PUqVMqy0xNTTHjxQs4asqzZ0BcnOpeECIiNeLT42H/pT3k+nL42vticY/FcKnrInUsnRBxJwIrzq7AhaQLeJD1ABsHbsRYr7FSx6rW2OBoofr162Ps2LHK++pOR//aZswA+vdXjE9JSVFcbTwnR3FIh4ioFL72vtgUsAlulm5IyUnBwoiF6PBdB1ydfBX1Tarh+D0tk52fDU9rT4xuNRqj94yWOo5OYIOjhfT09GBmZlY5K09MBEaMUFz52spKMQbn3DmeiI6IXqpvU9Xxe+0c2sFllQs2/7YZ09tXw/F7Wsa/qT/8myrO8TR271hpw+gINjhaKD09HV9++SX09fVhb2+PHj16oG7duppZ+fbtmlkPEdVoZnIzeFh74Oajajp+j3Qez4OjZezt7REQEICRI0eif//+yM7OxnfffYfc3FypoxERKT0reIa4tDjYmttKHYVILTY4WqZp06bw8PCAjY0NXFxc8Pbbb0MIgd9++03qaFVryRLAxweoXVtxKK1/f+DKFalTEdVYM47NwKmEU7idfhu/Jv6KIT8PQU5+Dsa04vg90k48RKXl5HI5rK2t8ejRI6mjVK3wcMX1nHx8FNPA//tfwM8PuHZNcRI/IqpSiZmJGLF7BNJy02BlaoV2Du1wLvAcHC04fo+0ExscLVdQUIC0tDQ4OTlJHeUvixcDc+cCU6YAa9ZUzjaOHlW9v3UrUKcOcOaMYm8OEVWp7UM4fo+qFzY4WubYsWNo1qwZ6tSpg5ycHERERCA/Px+tWrWSOprCuXPA+vVAy5ZVu92sLMWlGDQ12JqISItk52fj1mPFZWeKRBHuPrmLS8mXUM+4HhrV4WVnXgcbHC2TmZmJ3bt3Izc3F6ampnBwcEBgYCAsLCykjgY8eQKMHAl89x3w6adVu+2pUwEvL6B9+6rdLhFRFYh5EINum/864er88PmYHz4fY1qNwaaATdIFq8bY4GiZIUOGSB2hdBMmAEOGAN27V22DM306cPq04qavW9e+uXPnDs6ePYukpCRkZWVh4MCB8PLykjoWEVWxrk5dIebzsjOaxFlUVDbr1yuu2v3ZZ1W73WnTgJ9+AsLCFBcF1TH5+fmwtrZGnz59YGDAvzeIiDSFFZVe7fp1YM4cIDISkMurbrtTpypOTBgeDri5Vd12q1DTpk3RtGlTAMDevXslTkNEpDu4B4deLSpKcWkHT0/AwEBxO3UKWLdO8d95eepft3atYjBy7dqKW/v2wKFDZdvmlCnAxo2KvTd16wLJyYpbdrbm3hcREeks7sGhVwsIALy9VZeNGwc0barYs1PaXh0HB2DZMsXzioqAzZsV67pw4dWzsNatU/zbo4fq8vnzgQULXuttEBFRzcEGh17NwkJx+ztTU8UJ9zw9S3/dwIGq9xctAoKCFHuEXtXgCA62I9J1iyMXY27YXEzxmYI1/pV0Ti2qsdjgUNUoLAR27lQcYurQQeo0RCSxc4nnsD52PVraVPE5tajGYINDryc8vGzP+/13xdibZ88AMzNgzx6gRYtKjUZE2u3JsycY+ctIfDfgO3x6qorPqVUJlkQuwS9xv+B62nUYGhiinUM7LOmxBJ7WL9nDTZWOg4ypcrm6ApcuKc6APGkSMGYML5r5N/n5+UhOTkZycjKEEHjy5AmSk5Px5MkTqaMRVZoJBydgiPsQdHfuLnUUjQi/E47J3pNx9t2zCBsdBgM9A/ht8cPjp4+ljlajVWgPzsyZM3HgwAHI5XI0btwYGzdu1I4z7pL2kMuBJk0U/+3tDURHA199pTgbMuHBgwfYvHmz8n54eDjCw8PRqlUrBAQESJhMu7DW6I71F9bj1uNb2Dpoq9RRNOboKNVr520dtBV1ltbBmbtn0N+V186TSoUanJ49e2LJkiUwMDDAxx9/jCVLlmDZsmWayka6qKio9GnlNZCTkxPmz58vdQytx1qjG66nXcecsDmIHBcJuX4VnlOrimXlZaFIFKGuMa+dJ6UKNTi9evVS/ne7du2wa9euCgciHTJ7NtCvH9CwoeJimT/+qBi7U9Zz4RD9P9Ya3RCVGIW03DR4rvtrbEqhKETEnQh8E/MNcubkwNDAUMKEmjE1ZCq8GnihvQOvnScljQ0y/v777zFs2LBSHw8ODkZwcDAAIDU1VVObJW2WnAyMGqX4t04dxdTwI0eA3r2lTkbVGGtN9RXgFgBvO9Vzao3bNw5N6zXFnM5zdGKvzvSj03H67mmcHn8a+nq6de286uaVDY6fnx+Sk5NLLF+0aBEG/v95ThYtWgQDAwOMHDmy1PVMmDABEyZMAAB4v3jSONJNmzZJnYCqEdYa3WdhZAELI9WxU6a1TFHPuJ5OzDiaFjIN269ux8kxJ+FSV/eunVfdvLLBOX78+Esf37x5Mw4ePIgTJ05AJpNpLBhVvsjISISFhcHHxwf+/v5Sx6EajrWGqrOpR6Zi+9XtCB8TDjdL3bx2XnVToUNUISEhWLZsGU6dOgUTExNNZaIqkJiYiNjYWNjY2EgdheiVWGt0V/jYcI2vc+35tfj2wrdIyEgAAHhYe+CTzp+gX7N+Gt8WAEw5NAVbL2/F3uF7Ude4LpKzFXsizeRmMJObVco26dUq1OB88MEHyMvLQ8+ePQEoBv998803GglGlefZs2f45ZdfMGDAAJw6dUrqOESvxFpD5eFQ2wHL/Jahaf2mKBJF2HxpMwJ2BODChAuVcubkdTGKa+f12KJ67bz5XeZjQdcFGt8elU2FGpxbt25pKgdVoYMHD8Ld3R3Ozs5scKhaYK2h8hjopnodvEU9FiEoJghR96IqpcER83ntPG3EMxnXMBcuXMDjx4/RvbtunEGUiOhlCosKsf3KdmTnZ6NDQ14HrybhtahqkLS0NISFhWHcuHHQ1+f0xbKKjIxEXFwc0tLSYGBgAAcHB/To0QPW1tZSRyOiUvz+8He0/649nhU8g5ncDHuG7UELG14HryZhg1ODJCYmIjc3F+vWrVMuE0Lgzp07iImJwZw5c2BgwP8lXnTnzh14e3vD3t4eQgiEh4djy5YtmDJlCoyNjaWOR0RquFq64tLES8h4loHd13ZjzN4xCB8brhPT0als+NusBnFzc4OdnZ3Ksn379qFevXro3Lkz9+qUYtSoUSr3Bw0ahKVLl+Lu3btwdXWVKBURvYxcX44m9RTXwfO280b0g2h8FfUVvhvI6+DVFGxwahAjIyMYGRmpLKtVqxaMjY15uKUc8vLyIITg3huiaqRIFCGvkNfBq0nY4BCVU0hICBo0aAAHBwepoxCRGrOPz0a/pv3QsE5DZOVl4cfff0R4QjgOvc3r4NUkbHBquLFjx0odoVo5evQo7t69i/Hjx0NPj5MQibRRcnYyRu0ZheTsZNQxrIOWNi1xZOQR9G7C6+DVJGxwiMooJCQEV69exZgxY1C3bl2p4xBRKTYFbJI6AmkB/glKVAZHjhzBlStXMHr0aFhaWkodh4iIXoF7cEhS58+fx4ULF5CRkQEAsLa2RufOndGsWTOJk/3l0KFDuHz5MoYPHw5jY2NkZ2cDAORyOeRyucTpiIhIHTY4JKnatWvDz88P9evXhxACly5dwo4dOzBhwgStuRBoTEwMAGDLli0qy7t06YKuXbtKkIiIiF6FDQ5Jys3NTeV+jx49EBMTg3v37mlNgzN//nypIxARUTmxwSGtUVRUhGvXriE/Px8NGzaUOg4REVVjbHBIcg8fPsR3332HgoICyOVyDBs2TGv23hARUfXEBockZ2lpiYkTJ+LZs2e4du0a9u7di7Fjx/LsykRE9No4TZwkp6+vj3r16sHOzg5+fn5o0KABoqKipI5FRETVGBsc0jpCCBQWFkodg4iIqjE2OCSp48eP486dO8jIyMDDhw9x/PhxJCQkoEWLFlJHIyKiaoxjcEhS2dnZ2LNnD7Kzs2FoaAgbGxuMHDkSTZo0kToaERFVY2xwSFIBAQFSRyAiIh3EQ1RERESkc9jgEBERkc5hg0NEREQ6hw0OERER6RyNNDgrVqyATCZDWlqaJlZHRKQWaw0RlVWFG5x79+4hNDQUjRo10kQeIiK1WGuIqDwq3OBMmzYNy5cvh0wm00QeIiK1WGuIqDwqdB6c/fv3w97eHq1atXrlc4ODgxEcHAwASE1NrchmiaiGYa0hovJ6ZYPj5+eH5OTkEssXLVqExYsX49ixY2Xa0IQJEzBhwgQAgLe3dzljEpGuY60hIk16ZYNz/Phxtct///133L59W/kXVWJiItq0aYPz58+jQYMGmk1JRDqPtYaINOm1D1G1aNECKSkpyvtOTk6IiYmBpaWlRoIREQGsNUT0engeHCIiItI5GrvYZkJCgqZWRURUKtYaIioL7sEhIiIincMGh4iIiHQOGxwiIiLSOWxwiIiISOewwSEiIiKdwwaHiIiIdA4bHCIiItI5bHCIiIhI57DBISIiIp3DBoeIiIh0DhscIiIi0jlscIiIiEjnsMEhIiIincMGh4iIiHQOGxwiIiLSOWxwiIiISOewwSEiIiKdwwaHiIiIdA4bHCIiItI5bHCIiIhI57DBISIiIp3DBoeIiIh0DhscIiIi0jlscIiIiEjnVLjBWb16NVxdXeHh4YFZs2ZpIhMRUQmsNURUHgYVefHJkyexb98+XL58GYaGhkhJSdFULiIiJdYaIiqvCu3BCQoKwuzZs2FoaAgAsLa21kgoIqK/Y60hovKqUINz48YNREZGwtfXF126dEF0dHSpzw0ODoa3tze8vb2Rmppakc0SUQ3DWkNE5fXKQ1R+fn5ITk4usXzRokUoKChAeno6zp07h+joaAwdOhTx8fGQyWQlnj9hwgRMmDABAODt7a2B6ESkS1hriEiTXtngHD9+vNTHgoKC8Oabb0Imk+Ef//gH9PT0kJaWBisrK42GJCLdx1pDRJpUoUNUAQEBCAsLA6DYhZyfnw9LS0uNBCMiKsZaQ0TlVaFZVOPHj8f48ePh6ekJuVyOzZs3q91lTERUEaw1RFReFWpw5HI5tm3bpqksRERqsdYQUXnxTMZERESkc9jgEBERkc5hg0NEREQ6hw0OERER6Rw2OERERKRz2OAQERGRzmGDQ0RERDqHDQ4RERHpHDY4REREpHPY4BAREZHOYYNDREREOkcmhBBVvVFLS0s4OTlVeD2pqamwsrKqeCANYBb1mKV02pRHU1kSEhKQlpamgUSaoYlao4ufk6ZoUx5mUU8Xs5S5zohqrG3btlJHUGIW9ZildNqUR5uyaBtt+tloUxYhtCsPs6hXk7PwEBURERHpHDY4REREpHP0FyxYsEDqEBXRtm1bqSMoMYt6zFI6bcqjTVm0jTb9bLQpC6BdeZhFvZqaRZJBxkRERESViYeoiIiISOewwSEiIiKdU+0bnNWrV8PV1RUeHh6YNWuW1HEAACtWrIBMJpP0fCAzZ86Em5sbWrZsiUGDBiEjI6PKM4SEhMDV1RVNmjTB0qVLq3z7xe7du4du3brB3d0dHh4eWLVqlWRZihUWFqJ169b45z//KWmOjIwMDBkyBG5ubnB3d0dUVJSkebSZttUa1hkFbakzAGvNy0hSa6p0UrqGhYWFiR49eohnz54JIYR4+PChxImEuHv3rujVq5do1KiRSE1NlSzH0aNHxfPnz4UQQsyaNUvMmjWrSrdfUFAgXFxcxJ9//iny8vJEy5YtxdWrV6s0Q7EHDx6ICxcuCCGEyMzMFE2bNpUsS7EvvvhCjBgxQvTr10/SHKNHjxbr168XQgiRl5cn0tPTJc2jrbSt1rDOKGhTnRGCteZlpKg11XoPTlBQEGbPng1DQ0MAgLW1tcSJgGnTpmH58uWQyWSS5ujVqxcMDAwAAO3atUNiYmKVbv/8+fNo0qQJXFxcIJfLMXz4cOzbt69KMxSztbVFmzZtAADm5uZwd3fH/fv3JckCAImJiTh06BACAwMlywAAmZmZiIiIwLvvvgsAkMvlsLCwkDSTttK2WsM6o6BNdQZgrSmNVLWmWjc4N27cQGRkJHx9fdGlSxdER0dLmmf//v2wt7dHq1atJM3xou+//x59+/at0m3ev38fDRs2VN53cHCQ9IteLCEhARcvXoSvr69kGT766CMsX74cenrSfv3i4+NhZWWFcePGoXXr1ggMDEROTo6kmbSVNtUa1pm/aGudAVhr/k6qWmNQ6VuoID8/PyQnJ5dYvmjRIhQUFCA9PR3nzp1DdHQ0hg4divj4+Er9q+ZleRYvXoxjx45V2rbLk2XgwIHK/zYwMMDIkSOrLBcACDVnH5D6r83s7GwMHjwYK1euRO3atSXJcPDgQVhbW6Nt27YIDw+XJEOxgoICxMbGYvXq1fD19cXUqVOxdOlSfPbZZ5Lmkoo21RrWmbLRxjoDsNa8SLJaU+kHwSpR7969xcmTJ5X3XVxcREpKiiRZLl++LKysrISjo6NwdHQU+vr6omHDhiIpKUmSPEIIsWnTJtGuXTuRk5NT5ds+e/as6NWrl/L+4sWLxeLFi6s8R7H8/HzRq1cv8cUXX0iWQQghZs+eLezt7YWjo6OwsbERxsbGYuTIkZJkSUpKEo6Ojsr7ERERwt/fX5Is2k5bag3rjCptqzNCsNaoI1WtqdYNTlBQkJg3b54QQojr168LBwcHUVRUJHEqBUdHR0kH/x05ckS4u7tL1vA9f/5cODs7i/j4eOXgvytXrkiSpaioSLzzzjti6tSpkmy/NCdPnpR84F+nTp1EXFycEEKI+fPnixkzZkiaR1tpa61hndGeOiMEa83LSFFrtP4Q1cuMHz8e48ePh6enJ+RyOTZv3qwVuye1wQcffIC8vDz07NkTgGIA4DfffFNl2zcwMMCaNWvQu3dvFBYWYvz48fDw8Kiy7f/dmTNnsHXrVrRo0QJeXl4AgMWLF8Pf31+SPNpk9erVGDlyJPLz8+Hi4oKNGzdKHUkrsdaoxzqjirWmdFLUGl6qgYiIiHROtZ5FRURERKQOGxwiIiLSOWxwiIiISOewwSEiIiKdwwaHiIiIdA4bHCIiItI5bHCIiIhI5/wfkAgk+6M7y7gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pos1=dict(zip(range(10),W1.data.numpy().T))\n",
    "pos2=dict(zip(range(10),W2.data.numpy()))\n",
    "#\n",
    "fig = plt.figure(figsize=(8, 4),facecolor='white')\n",
    "ax = fig.add_subplot(121)\n",
    "for i in pos1:\n",
    "    x,y=pos1[i]\n",
    "    #plt.plot(x,y,marker='o',color='r')\n",
    "    plt.text(x,y,i,size=14,color='r')\n",
    "#\n",
    "for i in pos2:\n",
    "    x,y=pos2[i]\n",
    "    #plt.plot(x,y,marker='^',color='gray')\n",
    "    plt.text(x,y,i,size=14,color='gray')\n",
    "#plt.legend(loc=3)\n",
    "l=7\n",
    "plt.xlim(-l,l)\n",
    "plt.ylim(-l,l)\n",
    "#annotation\n",
    "#plt.plot(0,0,'ro')\n",
    "#plt.text(0,0.5,'center vector (W1)',ha='center',color='r')\n",
    "#plt.plot(0,0,marker='^',color='gray')\n",
    "#plt.text(0,-0.5,'context vector (W2)',ha='center',color='gray')\n",
    "plt.title('W1 (red) and W2 (gray)',fontsize=16)\n",
    "#\n",
    "ax = fig.add_subplot(122)\n",
    "for i in pos1:\n",
    "    x1,y1=pos1[i]\n",
    "    x2,y2=pos2[i]\n",
    "    #plt.plot((x1+x2)/2,(y1+y2)/2,marker='s',color='green')\n",
    "    plt.text((x1+x2)/2,(y1+y2)/2,i,size=14,color='green')\n",
    "\n",
    "plt.xlim(-l,l)\n",
    "plt.ylim(-l,l)\n",
    "plt.title('Averaged',fontsize=16)\n",
    "#\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "celltoolbar": "Slideshow",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
